#include "DicProducer.hh"
#include <iostream>
#include <fstream>
#include <dirent.h>
#include <sys/stat.h>
#include "utils.hh"
#include "SplitToolCppJieba.hh"
using namespace std;

// 处理单个文件的构造函数
// Dictproducer::Dictproducer(const string &filepath)
// {
//     std::ifstream file(filepath);
//     if (file)
//     {
//         this->_engFiles.push_back(filepath);
//     }
// }
// 处理文件号的构造函数
Dictproducer::Dictproducer(SplitTool *splitTool)
    : _splitTool(splitTool) {}

void Dictproducer::addFile(const string &fileOrDir, int mode)
{
    struct stat path_stat;
    // 判断dir是文件路径还是目录路径
    if (stat(fileOrDir.c_str(), &path_stat) == 0)
    {
        if (S_ISREG(path_stat.st_mode))
        {
            // dir传入的是一个文件
            std::ifstream file(fileOrDir);
            if (file)
            {
                if (mode == 1)
                    this->_engFiles.push_back(fileOrDir);
                else
                    this->_CNFiles.push_back(fileOrDir);
            }
        }
        else if (S_ISDIR(path_stat.st_mode))
        {
            // dir传入的是一个目录
            DIR *dirp = opendir(fileOrDir.c_str());
            // 判断文档路径是否正确
            if (dirp == NULL)
            {
                cout << "opendir() 打开失败...\n";
                return;
            }

            // 打开文档
            struct dirent *pdirent;
            while ((pdirent = readdir(dirp)) != NULL)
            {
                // 去掉.和..
                if (string(pdirent->d_name) == "." || string(pdirent->d_name) == "..")
                    continue;

                // 将文件的绝对路径存储_file中
                string filepath = fileOrDir + "/" + string(pdirent->d_name);
                if (mode == 1)
                    this->_engFiles.push_back(filepath);
                else
                    this->_CNFiles.push_back(filepath);
            }
            // 及时释放资源，关闭目录流
            closedir(dirp);
        }
    }
}

// 显示所有的文件路径
void Dictproducer::showFiles() const
{
    for (auto &file : _engFiles)
    {
        cout << "engFile:" << file << "\n";
    }

    for (auto &file : _CNFiles)
    {
        cout << "CNFile:" << file << "\n";
    }
}

// 创建停用词set
set<string> creatStopWordsSet(const string &stopWordsPath)
{
    ifstream ifsstop(stopWordsPath);
    // 停用词过滤
    set<string> stopWordsSet;
    string stopword;
    while (getline(ifsstop, stopword))
    {
        if (stopword[stopword.size() - 1] == '\r')
        {
            // cout << "最后一个是\\r\n";
            stopword.resize(stopword.size() - 1);
        }
        stopWordsSet.insert(stopword);
    }
    return stopWordsSet;
}
// 处理每一行
void dealLine(string &line)
{
    // Shenzhen-HK应该拆分成两个单词
    // 如果直接使用istringstream将一行内容拆分成单词
    // 再进行处理，那么会把这样的内容处理成一个单词
    // 所以直接先处理一行内容
    // 可以用空格替代符号或数字
    for (auto &ch : line)
    {
        if (!isalpha(ch))
        {
            ch = ' ';
        }
        else if (isupper(ch))
        {
            ch = tolower(ch);
        }
    }
}
void Dictproducer::buildEnDict()
{
    for (auto &filename : _engFiles)
    {
        // 判断文件是否能打开
        ifstream ifs(filename);
        if (!ifs)
        {
            std::cerr << "ifstream open file failed! \n";
            return;
        }

        // 创建停用词
        string stopWordsPath = Utils::readConf()["stop_words_eng.txt"];
        set<string> stopWordsSet = creatStopWordsSet(stopWordsPath);

        // 创建词典
        string line;
        map<string, int> tmpMap;
        while (getline(ifs, line))
        {
            dealLine(line); // 读到的每一行先处理【删去非字母、将大写转为小写】

            // 1. 解析每一行的内容
            istringstream iss(line);

            string word;
            while (iss >> word)
            {
                // 2. 判断该词是否是停用词
                if (stopWordsSet.count(word) != 0)
                {
                    // 改词为停用词
                    continue;
                }

                // 4.统计单词出现次数
                ++tmpMap[word];
            }
        }

        // 存到_dict中
        _dict.reserve(tmpMap.size());
        for (auto &item : tmpMap)
        {
            _dict.push_back(item);
        }
        ifs.close();
    }

    /* // 测试_dict
    for(auto &item : _dict)
    {
        cout << "key:" << item.first << "\tvalue:" << item.second << "\n";
    } */
};

void Dictproducer::buildCnDict()
{
    string cnStopWordsPath = Utils::readConf()["stop_words_zh.txt"];
    set<string> cnStopWordset = creatStopWordsSet(cnStopWordsPath);

    // 处理文件夹中的每一个文件
    map<string, int> tmpCnDictMap;
    for (auto &filepath : _CNFiles)
    {
        vector<string> words = _splitTool->cut(filepath);
        for (auto &word : words)
        {
            // 过滤停用词
            if (cnStopWordset.count(word) != 0)
                continue;
            ++tmpCnDictMap[word];
        }
    }

    // 存到_dict中
    _dict.reserve(_dict.size() + tmpCnDictMap.size());
    for (auto &item : tmpCnDictMap)
    {
        // cout << "first:" << item.first << "\tsecond:" << item.second << "\n";
        _dict.push_back(item);
    }

    /* // 测试_dict
    for(auto &item : _dict)
    {
        cout << "key:" << item.first << "\tvalue:" << item.second << "\n";
    } */
};

void Dictproducer::storeDict(const char *filepath) // 将词典写入文件
{
    // 将vector<pair<string,int>> _dict;存到文档中
    ofstream ofs(filepath, ios::app);
    if (!ofs)
    {
        cerr << "ofs(filename)...\n";
        return;
    }

    for (auto &item : _dict)
    {
        string str = item.first + " " + std::to_string(item.second) + "\n";
        ofs.write(str.c_str(), str.size());
    }
    ofs.close();
};
void Dictproducer::storeIndex(const char *filepath)
{
}

void Dictproducer::showDict() const // 查看词典，作为测试用
{

};

void Dictproducer::pushDict(const string &word) {

}; // 存储某个单词

void Dictproducer::createIndex()
{
    string dictIndexFilePath = Utils::readConf()["dictIndex.dat"];

    ofstream ofs(dictIndexFilePath);
    if (!ofs)
    {
        cerr << "ofs(dictIndex) fail...\n";
        return;
    }

    for (size_t j = 0; j < _dict.size(); ++j)
    {
        string word = _dict[j].first;
        // 判断这是是中文还是英文
        for (size_t i = 0; i < word.size();)
        {
            int num = Utils::nBytesCode(word[i]);
            if (num == 1)
            {
                // 英文
                _index[word.substr(i, 1)].insert(j);
                i += num;
            }
            else
            {
                // 中文
                _index[word.substr(i, num)].insert(j);
                i += num;
            }
        }
    }

    // 将map<string,set<int>> _index;写入文件
    string charIndex;
    for (auto &item : _index)
    {
        charIndex = item.first;
        // cout << "charIndex:" << charIndex << "\n";
        for (auto &value : item.second)
        {
            charIndex = charIndex + " " + std::to_string(value);
        }
        charIndex += "\n";
        ofs.write(charIndex.c_str(), charIndex.size());
    }
    ofs.close();
};
Dictproducer::~Dictproducer()
{
    if (_splitTool)
    {
        delete _splitTool;
        _splitTool = nullptr;
    }
};

int main(int argc, char const *argv[])
{

    Dictproducer dictCreate;
    // 增加英文语料
    dictCreate.addFile("/home/tk/review/project/data/yuliao/english.txt", 1);
    // 增加中文语料
    // dictCreate.addFile("/home/tk/review/project/data/yuliao/C3-Art0019.txt",2);
    // 增加文件夹的中文语料
    dictCreate.addFile("/home/tk/review/project/data/yuliao/art", 2);

    // dictCreate.showFiles();

    dictCreate.buildEnDict();
    dictCreate.buildCnDict();
    dictCreate.storeDict(Utils::readConf()["dict.dat"].c_str());

    // 创建索引
    dictCreate.createIndex();
    return 0;
}
// g++ -o DicProducer DicProducer.cpp SplitTool.cpp SplitToolCppJieba.cpp ../utils/utils.cpp  -I ../include/ -I ../utils